• Data integrity
• Reshaping
• Filtering
• Merging
• Summarizing

# Make tidyverse load quietly
options(tidyverse.quiet = TRUE)
library(tidyverse)   #< General use
library(tidyr)       #< Reshaping
library(wbstats)     #< World bank data.
library(countrycode) #< Country coding
library(assertthat)  #< Results checking
library(lubridate)   #< Date manipulations
requireNamespace('zoo')#< time series

for .RData files use load()

load("data/ebola.data.RData")

for .rds files use readRDS() and capture the results in a variable.

ebola <- readRDS("data/ebola.data.rds")

• mutate() - modify/add variables
• mutate_at() - modify a set of variables.
• mutate_if() - modify variables meeting a criteria
• transmute() - create a new set of variables based on previous.

Basic Usage

select(data, ...)

Over the next few examples we will explore the forms ... can take

the easiest is with variable names:

ebola.data %>% select(Country, `Case def.`, `Total cases`) %>% head()

You can select everything but a variable with the minus operator

ebola.data %>% select(-`Total cases`) %>% head()

You can select by variable position as well.

ebola.data %>% select(1:4) %>% head()

Use single colon : with variable names to select variables named and eveything in between:

ebola.data %>% select(Country:`Total cases`) %>% head()

selection helpers are also provided:

ebola.data %>% select(starts_with("Total")) %>% head()

The available helpers are:

• starts_with()
• ends_with()
• contains() - must match literally
• matches() - Regular expression match
• num_range() - numerical ranged variables with a prefix
• all_of() - must match all of given variables named in a vector.
• any_of() - select any variables present in given vector, but no error if not present.
• everything() - Matches all variables, useful when reordering variables.
• last_col() - The last column

You may use multiple forms together

ebola.data %>% select(last_col(), 2:3, `Total cases`) %>% head()

Subset data to only confirmed cases for Nigeria.

filter( ebola.data
      , Country == 'Nigeria'
      , `Case def.` == 'Confirmed'
      )

to perform an or use the single |

filter( ebola.data
      , (Country == 'Nigeria') | (Country == 'Sierra Leone')
      , `Case def.` == 'Confirmed'
      )

an alternate form would be to use %in%

filter( ebola.data
      , Country %in% c('Nigeria', 'Sierra Leone')
      , `Case def.` == 'Confirmed'
      )

sort() is the base version but again, less robust.

Arrange allows you to give sorting criteria.

`Confirmed Cases for Sierra Leone` %>% 
    arrange(`Country Report Date`, desc(`Case def.`)) %>% 
    head()

• Wide Data
  • multiple observations for one unit are in columns
• Long Data
  • multiple observations for one unit are in rows.

# Make subsets
confirmed <- ebola.data %>% filter(`Case def.` == 'Confirmed') %>% 
    select(-`Case def.`) %>% distinct() %>% 
    rename_at(vars(starts_with("total")), ~paste("Confirmed", .))
probable  <- ebola.data %>% filter(`Case def.` == 'Probable') %>% 
    select(-`Case def.`) %>% distinct() %>%  
    rename_at(vars(starts_with("total")), ~paste("Probable", .))
suspected <- ebola.data %>% filter(`Case def.` == 'Suspected') %>% 
    select(-`Case def.`) %>%  distinct() %>% 
    rename_at(vars(starts_with("total")), ~paste("Suspected", .))
all.cases <- ebola.data %>% filter(`Case def.` == 'All') %>% 
    select(-`Case def.`) %>%  distinct() %>% 
    rename_at(vars(starts_with("total")), ~paste("All", .))

Operations have these parameters:

• by - variables to join on, defaults to common variables
• suffix - sufixes to add to distinguish common variables that are not part of by

# spread each
cases <- ebola.data %>% select(-`Total deaths`) %>% 
    tidyr::spread('Case def.', 'Total cases')
deaths <-  ebola.data %>% select(-`Total cases`) %>% 
    tidyr::spread('Case def.', 'Total deaths')
ebola.option3 <- 
    full_join( cases, deaths
             , c('SheetName', 'Country', 'Country Report Date')
             , suffix = c(".cases", ".deaths")) 
glimpse(ebola.option3)

## Rows: 838
## Columns: 11
## $ SheetName             <chr> "Apr 01", "Apr 01", "Apr ...
## $ Country               <chr> "Guinea", "Liberia", "Mal...
## $ `Country Report Date` <dttm> 2015-03-29, 2015-03-29, ...
## $ All.cases             <int> 3492, 9712, 8, 20, 1, 119...
## $ Confirmed.cases       <int> 3068, 3151, 7, 19, 1, 854...
## $ Probable.cases        <int> 414, 1879, 1, 1, 0, 287, ...
## $ Suspected.cases       <int> 10, 4682, 0, 0, 0, 3142, ...
## $ All.deaths            <int> 2314, 4332, 6, 8, 0, 3799...
## $ Confirmed.deaths      <int> 1900, 0, 0, 7, 0, 3433, 0...
## $ Probable.deaths       <int> 414, 0, 0, 1, 0, 208, 0, ...
## $ Suspected.deaths      <int> 0, 0, 0, 0, 0, 158, 0, 0,...

summarize( ebola.option3
         , 'Observations' = n()
         , 'Number of countries' = n_distinct(Country)
         , "# of Reporting dates" = n_distinct(`Country Report Date`)
         , max.cases = max(All.cases, na.rm=TRUE)
         , max.deaths = max(All.deaths, na.rm=TRUE)
         )

ebola.option3 %>% group_by(Country) %>% 
  summarise( "# of Reporting dates" = n_distinct(`Country Report Date`)
           , max.cases = max(All.cases, na.rm=TRUE)
           , max.deaths = max(All.deaths, na.rm=TRUE)
           )

The wbstats package provides access to the world bank database.

library(wbstats)
wbsearch('population', extra=TRUE)

wb(indicator ="SP.POP.TOTL", startdate = 2014, enddate=2014)

Run these commands investigate the output and let’s discuss.

To merge the the world bank data to our ebola data we need a common country variable.

library(countrycode)
long.ebola <- mutate( ebola.option3
                    , iso3c = countrycode(Country, "country.name", "iso3c"))

Allways, allways, allways, check your results.

assert_that(!any(is.na(long.ebola$iso3c)))

## [1] TRUE

• SP.URB.TOTL.ZS - Percentage of Population in Urban Areas (in % of Total Population)
• SP.POP.TOTL.MA.ZS - Population, male (% of total)
• SP.POP.TOTL - Population, total
• EN.POP.DNST - Population density (people per sq km)
• IN.POV.HCR.EST.TOTL - Poverty HCR Estimates (%) - Total
• NY.GDP.PCAP.CD - GDP per capita (current US$)

pop.vars <- c( 'SP.URB.TOTL.ZS', 'SP.POP.TOTL.MA.ZS'
             , 'SP.POP.TOTL', 'EN.POP.DNST'
             , 'IN.POV.HCR.EST.TOTL', 'NY.GDP.PCAP.CD')
pop.data <- wb( country = unique(long.ebola$iso3c)
              , indicator = pop.vars
              , startdate = 2014, enddate=2014)

meta.pop.data <- select(pop.data, variable=indicatorID, label=indicator) %>% distinct()
our.data <- 
    pop.data %>% 
    select(iso3c, value, indicatorID) %>% 
    spread(indicatorID, value) %>% 
    right_join(long.ebola)

## Joining, by = "iso3c"

Create a table 1

• Restrict the data to the most recent only.
• Columns should be Africa, Other, and Total.
• Rows should be summaries of the variables we have.
  • minimum, median, mean, maximum …